/*------------------------------------------------------------------------------------------

  Author:: Shinobu TAKANASHI <sino@takanashi-it-factory.com>
  Copyright:: (c)2010 Takanashi IT Factory, Ltd. All rights reserved.
  License:: BSD
  
  This script is re-written version of the ruby script on http://bit.ly/aMuczN

 -------------------------------------------------------------------------------------------*/

var ExtractContent = {
    var default = {
	threshold : 100,
	min_length : 80
	decay_factor : 0.73,
	continuous_factor : 1.62,
	punctuation_weight : 10,
	punctuations : /([、。，．！？]|\.[^A-Za-z0-9]|,[^0-9]|!|\?)/g,
	waste_expressions : /Copyright|All Rights Reserved/i,
	debug : false,
    };
    var CHARREF = {
	'&nbsp;'  : ' ',
	'&lt;'    : '<',
	'&gt;'    : '>',
	'&amp;'   : '&',
	'&laquo;' : "\xc2\xab",
	'&raquo;' : "\xc2\xbb",
    };
    var set_default = function(opt){
	if(opt){
	    this.default = opt;
	}
    };
    // Analyses the given HTML text, extracts body and title.
    var analyse = function(html, opt){
	// frameset or redirect
	if (html.match(/<\/frameset>|<meta\s+http-equiv\s*=\s*["']?refresh['"]?[^>]*url/i)){
	    return ["", this.extract_title(html)]
	}

	//option parameters
	if (opt){
	    for(var p in opt){
		this.default[p] = opt[p];
	    }
	}
	opt = this.default;
		    
	
	threshold=min_length=decay_factor=continuous_factor=punctuation_weight=punctuations=waste_expressions=debug=null;
	for(var k in opt){
	    eval("var "+ k +" = opt['" + k + "']", b) ;
	}
	// header & title
	if (html.match(/<\/head\s*>/im)){
	    title = RegExp.leftContext;
	    html  = RegExp.rightContext;
	    this.extract_title(title)
	}else{
	    extract_title(html);
	}

	// Google AdSense Section Target
	html = html.replace(/<!--\s*google_ad_section_start\(weight=ignore\)\s*-->.*?<!--\s*google_ad_section_end.*?-->/ig, '');
	html = html.replace(/<!--\s*google_ad_section_start[^>]*-->.*?<!--\s*google_ad_section_end.*?-->/ig, '');
	// eliminate useless text
	html = this.eliminate_useless_tags(html)

	// h? block including title
	var htag_reg = /(<h\d\s*>\s*(.*?)\s*<\/h\d\s*>)/ig;
	var matched = html.match(htag_reg);
	for(var i = 0; i < matched.length; i++){
	    if(matched[i].match(htag_reg)){
		// alert(RegExp.$2.indexOf(title, 0));
		if(RegExp.$2.length >= 3 && RegExp.$2.indexOf(title, 0) >= 0){
		    alert(matched[i]);
		    html = html.replace(matched[i], "<div>" + RegExp.$2 + "</div>");
		}
	    }
	}


	//extract text blocks
	factor = continuous = 1.0
	body = ''
	score = 0
	bodylist = []
	list = html.split(/<\/?(?:div|center|td)[^>]*>|<p\s*[^>]*class\s*=\s*["']?(?:posted|plugin-\w+)['"]?[^>]*>/)
	for(var block in list){
	    if(! block) continue;
	    block = block.replace(/^\s+|\s+$/g, ""); //like strip in Ruby
            if(has_only_tags(block)) continue;
	    continuous = continuous / continuous_factor if body.length > 0

	    // リンク除外＆リンクリスト判定
	    notlinked = eliminate_link(block)
	    if(notlinked.length < min_length) continue;

	    // スコア算出
	    c = (notlinked.length + notlinked.match(punctuations).length * punctuation_weight) * factor;
	    factor = factor * decay_factor;
	    not_body_rate = block.match(waste_expressions).length + block.match(/amazon[a-z0-9\.\/\-\?&]+-22/ig).length / 2.0;
	    if(not_body_rate > 0){
		c = c  * (0.72 ** not_body_rate);
	    }
	    c1 = c * continuous;
	    // puts "----- #{c}*#{continuous}=#{c1} #{notlinked.length} \n#{strip_tags(block)[0,100]}\n" if debug
	    
	    # ブロック抽出＆スコア加算
	    if(c1 > threshold){
		body = body + block + "\n";
		score = store + c1;
		continuous = continuous_factor;
	    }else if(c > threshold){ // continuous block end
		bodylist.push([body, score]);
		body = block + "\n";
		score = c;
		continuous = continuous_factor;
            }

	}
	
	bodylist.push([body, score]);
	body = '';
	var max_score = 0;
	for(var a in bodylist){
	    if(a[1] >= max_score){
		max_score = a[1];
		body = a[0];
	    }
	}
	return([strip_tags(body), title]);
    };
    // Extracts title.
    var extract_title = function(st){
	if(st.match(/<title[^>]*>\s*(.*?)\s*<\/title\s*>/i)){
	    return(this.strip_tags(RegExp.$1))
	}else{
	    return("")
	    
	}
    };
    // Eliminates useless tags
    var eliminate_useless_tags = function(html){
	// eliminate useless symbols
	html = html.replace(/\342(?:\200[\230-\235]|\206[\220-\223]|\226[\240-\275]|\227[\206-\257]|\230[\205\206])/g,'')

	// eliminate useless html tags
	html = html.replace(/<(script|style|select|noscript)[^>]*>.*?<\/\1\s*>/gimn, '');
	html = html.replace(/<!--.*?-->/gm, '');
	html = html.replace(/<![A-Za-z].*?>/gs, '');
	html = html.replace(/<div\s[^>]*class\s*=\s*['"]?alpslab-slide["']?[^>]*>.*?<\/div\s*>/gm, '');
	html = html.replace(/<div\s[^>]*(id|class)\s*=\s*['"]?\S*more\S*["']?[^>]*>/gis, '');
	return(html);
    };

    // Checks if the given block has only tags without text.
    var has_only_tags function(st){
	if(st.replace(/<[^>]*>/gimn, '').replace("&nbsp;",'').replace(/^\s+|\s+$/g, "").length == 0){
	    return true;
	}else{
	    return false;
	}
    };

    // リンク除外＆リンクリスト判定
    var eliminate_link function(html){
	var count = 0;
	var link_regex = /<a\s[^>]*>.*?<\/a\s*>/gimn;
	notlinked = html.replace(link_regex, "");
	count += html.match(link_regex).length;
	notllinked = notlinked.replace(/<form\s[^>]*>.*?<\/form\s*>/imn, '');
	notlinked = strip_tags(notlinked);
	if(notlinked.length < 20 * count || this.islinklist(html)){
	    return("");
	}else{
	    return(notlinked);
	}
    };
    // リンクリスト判定
    // リストであれば非本文として除外する
    /*
    var islinklist = function(st){
	if(st.match(/<(?:ul|dl|ol)(.+?)<\/(?:ul|dl|ol)>/ism)){
	    listpart = RegExp.$1;
      outside = st.gsub(/<(?:ul|dl)(.+?)<\/(?:ul|dl)>/ismn, '').gsub(/<.+?>/mn, '').gsub(/\s+/, ' ')
      list = listpart.split(/<li[^>]*>/)
      list.shift
      rate = evaluate_list(list)
      outside.length <= st.length / (45 / rate)
    end
  end
    */
    // Strips tags from html.
    var strip_tags = function(html){
	st = html.replace(/<.+?>/m, '')
	// Convert from wide character to ascii
	st.gsub!(/\357\274([\201-\272])/){($1[0]-96).chr} # symbols, 0-9, A-Z
    st.gsub!(/\357\275([\201-\232])/){($1[0]-32).chr} # a-z
    st.gsub!(/\342[\224\225][\200-\277]/, '') # keisen
    st.gsub!(/\343\200\200/, ' ')
    self::CHARREF.each{|ref, c| st.gsub!(ref, c) }
    require 'cgi'
    st = CGI.unescapeHTML(st)
    st.gsub(/[ \t]+/, " ")
    st.gsub(/\n\s*/, "\n")
  end



    
};

